package com.andnnl.file;

import cn.hutool.core.collection.IterUtil;
import cn.hutool.core.util.ReUtil;
import com.andnnl.fileanalysis.AnalysisListener;
import com.andnnl.fileanalysis.AnalysisParam;
import com.andnnl.fileanalysis.FileAnalysisManage;
import com.andnnl.xiaoshuo.BufferedRandomAccessFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;


/**
 * 域名提取
 * Created by chenss on 2024/1/10.
 */
public class HttpWhiteDomainTest {
    private static final Logger logger = LoggerFactory.getLogger(HttpWhiteDomainTest.class);

    public static void main(String[] args) throws IOException {
//        test1();
        test2();
        return;
    }

    public static void test2() {
        File file = new File("C:\\Users\\Administrator\\Desktop\\whiteDomain.csv");
        AnalysisListener listener = rowList -> {
//            logger.info("batch size:" + rowList.size());
            for (List<String> strings : rowList) {
//                logger.info(strings.toString());
                final String uid = strings.get(0).trim();
                final String eprId = strings.get(1);
                final String domain = strings.get(2).replaceAll("\n", "");
                if (uid.indexOf(",") != -1) continue;
                Pattern regexDomain = Pattern.compile("((http://|https://){0,1}([\\w\\-~]+\\.)+([a-z]){2,6})(/.*)?", Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.MULTILINE);
                List<String> domainList = ReUtil.findAll(regexDomain, domain, 1);//正则获取域名列表
                domainList=domainList.stream().filter(d->d.indexOf(".html")==-1).distinct().collect(Collectors.toList());
//                logger.info("domainList:" + domainList);
//                System.out.println();
                if(IterUtil.isNotEmpty(domainList)){
                    System.out.println(eprId+","+uid+","+0+","+domainList.stream().collect(Collectors.joining(";")));
                }
            }
        };
        AnalysisParam param = AnalysisParam.build()
                .setBatchSize(100000)//读到一批后调用一次invokeBatch
                .setDelHeadRows(1)//删除头部行数，空行不算
//                .setExt("txt")//指定解析的文件格式
//                .setCharset("UTF-8")//指定解析的文件编码
//                .setCharset("GBK")//指定解析的文件编码
//                .setSplitLimit(3)//txt分割2列
                .setMaxRow(0);//限制读取行数,0:不限制
//        CountListener countlistener=new CountListener();
//        FileAnalysisManage.analysis(file, countlistener, param);
//        logger.info("listener.count:"+countlistener.getCount());
        FileAnalysisManage.analysis(file, listener, param);
    }

    private static void test1() throws IOException {
        BufferedRandomAccessFile ra = new BufferedRandomAccessFile("C:\\Users\\Administrator\\Desktop\\whiteDomain.csv", "r");
        String line;
        while ((line = ra.readLine()) != null) {
//                offsetIndex[i++] = ra.getFilePointer();
            System.out.println(line);
        }
        ra.close();
    }
}
